package com.spider.huxiu;

import java.nio.charset.Charset;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashSet;
import java.util.Set;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.EntityUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import com.google.gson.Gson;
import com.spider.bean.Article;
import com.spider.bean.ResponseResule;
import com.spider.dao.ArticleDao;

/**
 * 
 * @ClassName: HuXiuSpiderQueue
 * @Description: 虎嗅新闻网爬虫 ,采用队列,多线程的方式爬取分页,及文章详情
 * @author Administrator
 * @date 2018年1月30日 下午7:22:49
 */
public class HuXiuSpiderQueueLinux {

	private static Set<String> ARTICLE_IDS = new HashSet<String>();
	private static int TOTAL_PAGE = 10;
	private static int ARTICLE_COUNT = 0;

	private static final String HUXIU_HUXIU_HASH_CODE = "bc1acc4ae8cc354069293a255b8140fc";
	private static final int THREAD_POOL_COUNT = 15;// 线程池中线程数量
	private static final String HUXIU_URL = "https://www.huxiu.com";
	private static final ArrayBlockingQueue<String> ARRAY_BLOCKING_QUEUE = new ArrayBlockingQueue<String>(500);
	private static final ExecutorService THREAD_POOL = Executors.newFixedThreadPool(THREAD_POOL_COUNT);
	private static final ArticleDao ARTICLE_DAO = new ArticleDao();

	public static void main(String[] args) {
		try {

			String data_last_dateline = pickHUXIU_HomePage();
			for (int i = 1; i <= THREAD_POOL_COUNT; i++) {
				THREAD_POOL.execute(new Runnable() {
					public void run() {
						while (true) {
							pickArticleDetailAndSave2DB();
						}
					}
				});
			}

			for (int pageIndex = 2; pageIndex < TOTAL_PAGE; pageIndex++) {
				ARTICLE_COUNT += ARTICLE_IDS.size();
				System.out.println("\r\n\t\t正在解析第" + (pageIndex - 1) + "页\t本页共" + ARTICLE_IDS.size() + "条链接,一共"
						+ ARTICLE_COUNT + "条链接\r\n");
				ARTICLE_IDS.clear();
				data_last_dateline = pickPageingUrlByHttpPostAndPasreResultJson(data_last_dateline, pageIndex + "");
			}
		} catch (Exception e) {
		}

	}

	private static String pickHUXIU_HomePage() {
		try {
			// 爬取网页
			Document document = pickPagingArticleByHttpGet(HUXIU_URL);
			Elements elements = document.select(".mod-info-flow>div");
			String data_last_dateline = document.select("div.get-mod-more").get(0).attr("data-last_dateline").trim();
			if (data_last_dateline != null && elements.size() >= 1) {
				// 得到h2 标签从而中文章链接,文章标题
				Elements selects = null;
				// 得到文章内部地址,后面拼接网址
				String articleHref = null;

				String articleTitle = null;

				// 遍历每一个文章的DIV
				for (Element element : elements) {

					// 获取判断第一个div的值是否为热点
					Elements hotTitle = element.select("div:eq(0)").select("div:eq(0)");

					String text = hotTitle.text();

					if (text.startsWith("热")) {
						selects = element.select("div:eq(3)").select("h2").select("a");
					} else {
						selects = element.select("div:eq(2)").select("h2").select("a");
					}
					articleHref = selects.attr("href");
					articleTitle = selects.text().trim();
					String articleUrl = HUXIU_URL + articleHref;
					ARTICLE_IDS.add(articleTitle + "xxc|" + articleUrl
							+ new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date())
							+ "|By熊骑士xiongxiaochuan_work@foxmail.com");
					// 添加文章链接到队列中
					ARRAY_BLOCKING_QUEUE.put(articleUrl);
				}
			}
			return data_last_dateline;
		} catch (Exception e) {
		}
		return null;
	}

	private static void pickArticleDetailAndSave2DB() {
		try {
			String articleUrl = null;
			if (ARRAY_BLOCKING_QUEUE.size() >= 1) {
				articleUrl = ARRAY_BLOCKING_QUEUE.take();
				System.out.println("线程小弟:" + Thread.currentThread().getId() + "\t正在处理文章链接:" + articleUrl);
				// 保存到数据库
				article2DB(articleUrl.trim());
			}
		} catch (Exception e) {
		}

	}

	private static void article2DB(String articleUrl) {
		try {
			Document document = pickPagingArticleByHttpGet(articleUrl);
			int lastIndexOf = articleUrl.lastIndexOf("/") + 1;
			String id = articleUrl.substring(lastIndexOf, lastIndexOf + 6);
			Elements elements = document.select(".article-wrap");
			String articleAuthName = elements.get(0).select(".author-name").text().trim();
			String articlePublishTime = elements.get(0).select(".article-time").text().trim();
			String articleTitle = elements.get(0).select(".t-h1").text().trim();
			Elements articleContent = elements.get(0).select(".article-content-wrap>p");
			articleTitle = articleTitle.replace(" ", "").replace("?", "？").replace("\\", "分");

			Article article = new Article();
			article.setArticleTitle(articleTitle);
			article.setArticleAuthName(articleAuthName);
			article.setArticleContent(articleContentToString(articleContent));
			article.setArticlePublishTime(articlePublishTime);
			article.setArticleUrl(articleUrl);
			article.setId(id);
			// 保存到数据库
			ARTICLE_DAO.save(article);
		} catch (Exception e) {
			System.out.println("线程小弟:" + Thread.currentThread().getId() + "\t解析" + articleUrl + "失败");
		}
	}

	private static String articleContentToString(Elements articleContent) {
		try {
			StringBuilder stringBuilder = null;
			if (articleContent != null) {
				stringBuilder = new StringBuilder();
				for (Element element : articleContent) {
					stringBuilder.append(element.html().trim());
				}
				return stringBuilder.toString();
			}
		} catch (Exception e) {
		}
		return null;
	}

	private static Document pickPagingArticleByHttpGet(String url) {
		try {
			CloseableHttpClient httpClient = HttpClients.createDefault();
			HttpGet HttpGet = new HttpGet(url);
			HttpGet.addHeader("User-Agent",
					"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.110 Safari/537.36");
			CloseableHttpResponse response = httpClient.execute(HttpGet);
			String html = EntityUtils.toString(response.getEntity(), Charset.forName("UTF-8"));
			return Jsoup.parse(html);
		} catch (Exception e) {
		}
		return null;
	}

	private static String pickPageingUrlByHttpPostAndPasreResultJson(String data_last_dateline, String pageIndex) {
		try {
			String pageingUrl = "https://www.huxiu.com/v2_action/article_list";
			CloseableHttpClient httpClient = HttpClients.createDefault();
			HttpPost httpPost = new HttpPost(pageingUrl);
			ArrayList<BasicNameValuePair> parameters = new ArrayList<BasicNameValuePair>();
			parameters.add(new BasicNameValuePair("huxiu_hash_code", HUXIU_HUXIU_HASH_CODE));
			parameters.add(new BasicNameValuePair("page", pageIndex));
			parameters.add(new BasicNameValuePair("last_dateline", data_last_dateline));
			httpPost.setEntity(new UrlEncodedFormEntity(parameters));
			httpPost.addHeader("User-Agent",
					"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.110 Safari/537.36");
			CloseableHttpResponse response = httpClient.execute(httpPost);
			String responseResultJson = EntityUtils.toString(response.getEntity(), Charset.forName("UTF-8"));
			Gson gson = null;
			String last_dateline = null;
			String articleDivDatas = null;
			if (responseResultJson != null) {
				gson = new Gson();
				ResponseResule responseResule = gson.fromJson(responseResultJson, ResponseResule.class);
				if (responseResule != null) {
					last_dateline = responseResule.getLast_dateline();
					articleDivDatas = responseResule.getData();
					TOTAL_PAGE = Integer.parseInt(responseResule.getTotal_page());
				}
			}
			if (articleDivDatas != null) {
				Document documents = Jsoup.parse(articleDivDatas);
				Elements elements = documents.select(".mob-ctt");
				String articleUrl = null;
				String articleTitle = null;
				if (elements.size() >= 1) {
					for (Element element : elements) {
						articleUrl = element.select("h2").select("a").get(0).attr("href").trim();
						articleTitle = element.select("h2").get(0).select("a").text().trim();
						ARRAY_BLOCKING_QUEUE.put(HUXIU_URL + articleUrl);
						String Url = articleTitle + "xxc|" + HUXIU_URL + articleUrl
								+ new SimpleDateFormat("|yyyy-MM-dd HH:mm:ss").format(new Date())
								+ "|By熊骑士xiongxiaochuan_work@foxmail.com";
						ARTICLE_IDS.add(Url);
					}
				}
			}
			return last_dateline;
		} catch (Exception e) {
			System.out.println("线程中断异常发生...");
			return null;
		}
	}
}
